package org.nerve.sprider.crawler4j;

import com.j256.ormlite.dao.Dao;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import org.nerve.sprider.common.DBUtil;
import org.nerve.sprider.common.StringUtil;
import org.nerve.sprider.domain.WebPage;

import java.util.Date;
import java.util.Set;
import java.util.regex.Pattern;

/**
 * org.nerve.study.sprider.crawler4j
 * Created by zengxm on 2016/3/17 0017.
 */
public class CommonCrawler extends WebCrawler {
	protected final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg"
			+ "|png|mp3|mp3|zip|gz))$");

	private String[] myCrawlDomains;

	private Dao<WebPage, Long> dao;

	public CommonCrawler(){
		super();

		try{
			dao= DBUtil.getDao(WebPage.class);
		}catch (Exception e){
			e.printStackTrace();
		}
	}

	@Override
	public void onStart() {
		myCrawlDomains = (String[]) myController.getCustomData();
	}

	@Override
	public boolean shouldVisit(Page referringPage, WebURL url) {
		String href = url.getURL().toLowerCase();
		if (FILTERS.matcher(href).matches()) {
			return false;
		}
		for (String crawlDomain : myCrawlDomains) {
			if (href.startsWith(crawlDomain)) {
				return true;
			}
		}
		return false;
	}

	@Override
	protected void onUnhandledException(WebURL webUrl, Throwable e) {
		super.onUnhandledException(webUrl, e);

		//
		logger.error("UnhandledException:"+e.getMessage());
	}

	@Override
	public void visit(Page page) {
		WebPage wp=new WebPage();
		WebURL wu=page.getWebURL();

		wp.setUrl(wu.getURL());
		wp.setDomain(wu.getDomain());
		wp.setAddDate(new Date());

		logger.info("->crawl："+wu.getURL());

		if (page.getParseData() instanceof HtmlParseData) {
			HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
			wp.setTitle(htmlParseData.getTitle());//获取网页标题
			String text = htmlParseData.getText();
			String html = htmlParseData.getHtml();
			Set<WebURL> links = htmlParseData.getOutgoingUrls();

			wp.setText(text)
					.setTextLen(text.length())
					.setHtml(html)
					.setHtmlLen(html.length())
					.setHtmlMd5(StringUtil.md5(html))
					.setLinkCount(links.size())
					;

			try{
				if(dao!=null){
					dao.create(wp);
					logger.info("-->save:"+wp.getUrl());
				}
			}catch (Exception e){
				logger.error("error on save data:"+e.getMessage()+", url="+wp.getUrl());
				e.printStackTrace();
			}
		}
	}

	protected void onVisited(Page page){
	}
}
